﻿using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Xml.Linq;
using System.Xml.XPath;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;

namespace FBGraph.ClientGenerator
{
    internal class Parser : IDisposable
    {
        // detailed data is not retrieved for these connections because their id properties are not self-referencing
        private static readonly String[] UndetailedConnectionNames = new String[] {
            "picture",
            "likes",
            "attending",
            "declined",
            "invited",
            "maybe",
            "noreply",
            "tagged",
            "tags",
            "members",
            "activities", 
            "interests",
            "music",
            "books",
            "movies",
            "television"
        };

        private class ParserQueueItem
        {
            public ParserQueueItem(String status, String url, Boolean useDocToken, Int32 retries, FileInfo cachedFile, Action<JObject, Uri> callback)
            {
                this.Callback = callback;
                this.UseDocToken = useDocToken;
                this.Retries = retries;
                this.CachedFile = cachedFile;
                this.Status = status;
                this.Url = url;
            }

            public Boolean UseDocToken { get; set; }
            public Int32 Retries { get; set; }
            public Action<JObject, Uri> Callback { get; set; }
            public HttpWebRequest Request { get; set; }
            public FileInfo CachedFile { get; set; }
            public String ErrorContent { get; set; }
            public String Status { get; set; }
            public String Url { get; set; }
        }

        public Parser()
        {
            this.Queue = new Queue<ParserQueueItem>();
            webClient = new WebClient();
        }
        
        private WebClient webClient = null;

        private Queue<ParserQueueItem> Queue { get; set; }

        void IDisposable.Dispose()
        {
            webClient.Dispose();
        }

        private const String DOC_ROOT = "http://developers.facebook.com/docs/reference/api/";
        private const String DOC_PAGES_PATH = "/body/div[@class='body washbody reference']/div/div[@class='bodyText']/div[@class='refindex']/div[@class='page']";

        /// <summary>Generates an array of <see cref="ObjectData" /> objects with information about Graph API objects and the location of their detailed documentation pages</summary>
        internal ObjectData[] GetObjectData()
        {
            // download the root documentation page
            var content = webClient.DownloadString(DOC_ROOT);
            var doc = GetXmlFromDocumentationPage(content);
            var nav = doc.CreateNavigator();

            // get all the listed documentation pages
            var query =
                from node in nav.Select(DOC_PAGES_PATH).Cast<XPathNavigator>()
                select new
                {
                    Title = node.Select("div[@class='title']").Cast<XPathNavigator>().Single(),
                    Snippet = node.Select("div[@class='snippet']").Cast<XPathNavigator>().Single()
                } into bits
                select new ObjectData
                {
                    Name = bits.Title.Select("a").Cast<XPathNavigator>().Single().Value,
                    DetailUrl = DOC_ROOT + bits.Title.Select("a/@href").Cast<XPathNavigator>().Single().Value,
                    Description = bits.Snippet.Select("p").Cast<XPathNavigator>().Single().Value
                };

            var objectData = query.ToArray();

            PopulateDetailedObjectData(ref objectData);
            PopulateUserPropertyData(ref objectData);
            PopulateNonUserPropertyData(ref objectData);
            PopulateUserConnectionsData(ref objectData);
            PopulateNonUserConnectionsData(ref objectData);
            PopulateConnectionsObjectDetailData(ref objectData);
            // PopulatePages(ref objectData);
            return objectData;
        }

        /// <summary>Cleans the HTML markup from Facebook's documentation and parses it into a XML document for querying</summary>
        /// <param name="content">The raw HTML markup from Facebook's documentation site</param>
        /// <returns>An <see cref="XDocument" /> containing the content from the page's &lt;body&gt; tags.</returns>
        private XDocument GetXmlFromDocumentationPage(String content)
        {
            // the xml parser doesn't like some of the outer bits, so we string away everything outside of the body tags
            content = Regex.Match(content, "<body.*?>(.*)</body>", RegexOptions.Singleline).Captures[0].Value;

            // replaces HTML entities, which cause xml parsing to fail
            content = Regex.Replace(content, @"&\w+;", String.Empty);

            // add the xml dec
            content = "<?xml version=\"1.0\" ?>" + Environment.NewLine + content;

            var doc = XDocument.Parse(content);
            return doc;
        }

        private const String DETAIL_CONTENT_ROOT = "/body/div[@class='body washbody']/div[@class='content']/div[@class='bodyText']";
        private const String DETAIL_DESCRIPTION_PATH = "h2[1]/preceding-sibling::p";
        private const String DETAIL_EXAMPLE_PATH = "h2[1]/following-sibling::p[1]";
        private const String DETAIL_PROPERTIES_PATH = "table[1]/tr";
        private const String DETAIL_CONNECTIONS_PATH = "table[2]/tr";

        private void PopulateDetailedObjectData(ref ObjectData[] objectData)
        {
            foreach (var obj in objectData)
            {
                // download the object's documentation page
                var content = webClient.DownloadString(obj.DetailUrl);
                var doc = GetXmlFromDocumentationPage(content);
                var nav = doc.CreateNavigator();
                
                var contentRoot = nav.Select(DETAIL_CONTENT_ROOT).Cast<XPathNavigator>().Single();

                var description = contentRoot.Select(DETAIL_DESCRIPTION_PATH).Cast<XPathNavigator>();
                obj.Description = description.First().Value;
                obj.Remarks = description.Skip(1).Select(d => d.Value).ToArray();

                var exampleContent = contentRoot.Select(DETAIL_EXAMPLE_PATH).Cast<XPathNavigator>().Single();
                var exampleAnchor = exampleContent.Select("a[1]").Cast<XPathNavigator>().Single();
                obj.ExampleUrl = exampleAnchor.Select("@href").Cast<XPathNavigator>().Single().Value;
                obj.ExampleUrlText = exampleContent.Value;
                var exampleDescriptionAnchor = exampleContent.Select("a[2]").Cast<XPathNavigator>().Single();
                obj.ExampleDescription = exampleDescriptionAnchor.Value;
                obj.ExampleDescriptionUrl = exampleDescriptionAnchor.Select("@href").Cast<XPathNavigator>().Single().Value;

                var properties =
                    from node in contentRoot.Select(DETAIL_PROPERTIES_PATH).Cast<XPathNavigator>()
                    select new ObjectData.PropertyData
                    {
                        Name = node.Select("td[1]/code").Cast<XPathNavigator>().Single().Value,
                        Description = node.Select("td[2]/p").Cast<XPathNavigator>().Single().Value
                    };
                obj.Properties = properties.ToList();

                var connections =
                    from node in contentRoot.Select(DETAIL_CONNECTIONS_PATH).Cast<XPathNavigator>()
                    select new 
                    {                        
                        Anchor = node.Select("td[1]/code/a").Cast<XPathNavigator>().Single(),
                        Description = node.Select("td[2]/p").Cast<XPathNavigator>().Single()
                    } into node
                    select new ObjectData.ConnectionData
                    {
                        Name = node.Anchor.Value,
                        ExampleUrl = node.Anchor.Select("@href").Cast<XPathNavigator>().Single().Value,
                        Description = node.Description.Value
                    };
                obj.Connections = connections.ToList();
            }
        }

        private void PopulateUserPropertyData(ref ObjectData[] objectData)
        {
            var userData = objectData.Single(d => d.Name == "User");

            GetSampleData("Requesting user property data", "https://graph.facebook.com/me", false, (obj, source) =>
            {
                foreach (var property in obj)
                {
                    var userDataProperty = userData.Properties.SingleOrDefault(p => p.Name == property.Key);
                    if (userDataProperty == null)
                    {
                        userDataProperty = new ObjectData.PropertyData
                        {
                            Name = property.Key,
                            ExampleData = new ExampleData(property.Value, source)
                        };
                        userData.Properties.Add(userDataProperty);
                    }
                    else userDataProperty.ExampleData = new ExampleData(property.Value, source);
                }
            });

            ExecuteQueue();
        }

        private void PopulateNonUserPropertyData(ref ObjectData[] objectData)
        {
            foreach (var data in objectData.Where(o => o.Name != "User" && o.Name != "Status message"))
            {
                GetSampleData(String.Format("Requesting {0} property data", data.Name), data.ExampleUrl, true, (obj, source) =>
                {
                    foreach (var property in obj)
                    {
                        var dataProperty = data.Properties.SingleOrDefault(p => p.Name == property.Key);
                        if (dataProperty == null)
                        {
                            dataProperty = new ObjectData.PropertyData
                            {
                                Name = property.Key,
                                ExampleData = new ExampleData(property.Value, source)
                            };
                            data.Properties.Add(dataProperty);
                        }
                        else dataProperty.ExampleData = new ExampleData(property.Value, source);
                    }
                });
            }

            ExecuteQueue();
        }

        private void PopulateUserConnectionsData(ref ObjectData[] objectData)
        {
            var userData = objectData.Single(d => d.Name == "User");
            foreach (var connectionData in userData.Connections.Where(c => c.Name != "picture"))
            {
                GetSampleData(String.Format("Requesting user {0} connection data", connectionData.Name), String.Format("https://graph.facebook.com/me/{0}", connectionData.Name), false, (obj, source) =>
                    {
                        connectionData.ExampleData = new ExampleData(obj, source);
                    });
            }

            ExecuteQueue();
        }

        private void PopulateConnectionsObjectDetailData(ref ObjectData[] objectData)
        {
            // I used several linq queries here because it's complicated and spreading them out makes it easier to read
            
            // get all the connections across all objects
            // pictures are ignored because they're the actual pictures (binary data)
            // likes are ignored because they don't have their own ids (the id is the post itself)
            var allConnections =
                from obj in objectData
                from conn in obj.Connections
                where !UndetailedConnectionNames.Contains(conn.Name) && conn.ExampleData.Data != null
                select conn;

            // get array of example data for each connection
            var allConnectionData =
                from conn in allConnections
                select new { Connection = conn, Data = (JArray)conn.ExampleData.Data["data"] };

            // get the id of each data item
            var allValues =
                from connData in allConnectionData
                from obj in connData.Data

                // this bit of beauty thanks to http://bugs.developers.facebook.com/show_bug.cgi?id=10298
                // a users' "profile pictures" album apparently doesn't get its own id, and instead returns the user's uid - this needs to be ignored so
                // it doesn't pollute the user model
                select new { Connection = connData.Connection, IdObj = (JValue)obj["id"], FromObj = obj["from"] } into sometimesihateyoufacebook
                where
                    sometimesihateyoufacebook.IdObj != null &&
                    sometimesihateyoufacebook.IdObj.Value != null && (
                    sometimesihateyoufacebook.FromObj == null ||
                    !sometimesihateyoufacebook.FromObj.HasValues || (
                        ((JValue)sometimesihateyoufacebook.FromObj["id"]).Value.ToString() != sometimesihateyoufacebook.IdObj.Value.ToString()))

                select new { Connection = sometimesihateyoufacebook.Connection, Value = sometimesihateyoufacebook.IdObj.Value.ToString() };

            var allValuesWithTypes =
                from connValue in allValues
                where connValue.Value != null
                select new { Connection = connValue.Connection, ConnectionType = connValue.Connection.Name.ToPascalCase().Depluralize(), Value = connValue.Value };

            // group the ids by connection type
            var groupedValues =
                from connValue in allValuesWithTypes
                group new { Connection = connValue.Connection, Value = connValue.Value } by connValue.ConnectionType;

            foreach (var valueGroup in groupedValues)
            {
                var idList = String.Join(",", valueGroup.Select(g => g.Value).Take(250).ToArray());
                if (idList.Length > 0)
                {
                    GetSampleData(String.Format("Requesting {0} connection data", valueGroup.Key), String.Format("https://graph.facebook.com/?ids={0}", idList), false, (sampleData, source) =>
                    {
                        if (sampleData != null)
                        {
                            valueGroup.First().Connection.ExampleDetailData.AddRange(((IEnumerable<KeyValuePair<String, JToken>>)sampleData).Select(p => new ExampleData(p.Value, source)));
                        }
                    });
                }
            }

            ExecuteQueue();
        }

        private void PopulateNonUserConnectionsData(ref ObjectData[] objectData)
        {
            foreach (var data in objectData.Where(d => d.Name != "User"))
            {
                foreach (var connectionData in data.Connections.Where(c => c.Name != "picture"))
                {
                    GetSampleData(String.Format("Requesting {0} connection data", connectionData.Name), connectionData.ExampleUrl, true, (obj, source) =>
                    {
                        connectionData.ExampleData = new ExampleData(obj, source);
                    });
                }
            }

            ExecuteQueue();
        }

        private static readonly String[] PageSearchQueries = new String[]
        {
            "foo",
            "bar",
            "microsoft",
            "apple",
            "barack obama",
            "radiohead",
            "george clooney",
            "mini cooper",
            "gucci",
            "harvard",
            "heidi klum",
            "arrested development",
            "independence day"
        };

        private void PopulatePages(ref ObjectData[] objectData)
        {
            var userObj = objectData.Single(obj => obj.Name == "User");
            var interestsConn = userObj.Connections.Single(conn => conn.Name == "interests");

            var queue = new List<String>();

            foreach (var query in PageSearchQueries)
            {
                GetSampleData("Searching pages...", String.Format("https://graph.facebook.com/search?q={0}&type=page&limit=100", query), false, (obj, source) =>
                {
                    var ids =
                        from item in (JArray)obj["data"]
                        select ((JValue)item["id"]).Value.ToString();

                    queue.AddRange(ids);
                });
            }

            ExecuteQueue();

            while (queue.Count > 0)
            {
                var ids = queue.Take(300).ToArray();
                queue.RemoveRange(0, queue.Count >= 300 ? 300 : queue.Count);
                GetSampleData("Getting detailed page data...", String.Format("https://graph.facebook.com/?ids={0}", String.Join(",", ids)), false, (obj, source) =>
                {
                    interestsConn.ExampleDetailData.AddRange(((IEnumerable<KeyValuePair<String, JToken>>)obj).Select(p => new ExampleData(p.Value, source)));
                });
            }

            ExecuteQueue();
        }

        Int32 waiting = 0;
        private void ExecuteQueue()
        {
            while (this.Queue.Count > 0)
            {
                var item = this.Queue.Dequeue();
                item.Retries++;

                if (item.UseDocToken && String.IsNullOrEmpty(SavedAccessToken.Default.DocsToken))
                {
                    var docAuth = new DocAuth();
                    docAuth.ShowDialog();
                }

                var urlBuilder = new UriBuilder(item.Url);
                var queryBuilder = new StringBuilder((urlBuilder.Query ?? String.Empty).TrimStart('?'));

                // add the correct access token to the url
                queryBuilder.AppendFormat("{1}access_token={0}",
                    item.UseDocToken ? SavedAccessToken.Default.DocsToken : SavedAccessToken.Default.GeneratorToken,
                    queryBuilder.Length == 0 ? String.Empty : "&");
                urlBuilder.Query = null;
                urlBuilder.Query = queryBuilder.ToString();

                item.Request = (HttpWebRequest)WebRequest.Create(urlBuilder.Uri);
                item.Request.BeginGetResponse(new AsyncCallback(ExecuteQueueCallback), item);
                waiting++;

                Console.WriteLine();
                Console.WriteLine(String.Format("{0}: {1}", item.Status, item.Request.RequestUri));
                if (this.Queue.Count > 0) Thread.Sleep(250);
            }

            WaitForQueue();
        }

        private void WaitForQueue()
        {
            Console.WriteLine("Waiting on {0} requests...", waiting);

            while (waiting > 0)
            {
                if (this.Queue.Count > 0)
                {
                    ExecuteQueue();
                    return;
                }
                Thread.Sleep(0);
            }
        }

        private void ExecuteQueueCallback(IAsyncResult result)
        {
            waiting--;
            Console.WriteLine("Waiting on {0} requests...", waiting);
            var item = (ParserQueueItem)result.AsyncState;

            String content;
            try
            {
                var response = (HttpWebResponse)item.Request.EndGetResponse(result);

                using (var reader = new StreamReader(response.GetResponseStream()))
                {
                    content = reader.ReadToEnd();
                }
            }
            catch (WebException ex)
            {
                if (ex.Status == WebExceptionStatus.Timeout && item.Retries < MAX_RETRIES)
                {
                    this.Queue.Enqueue(item);
                    return;
                }

                if (item.UseDocToken && ((HttpWebResponse)ex.Response).StatusCode == HttpStatusCode.InternalServerError && item.Retries < MAX_RETRIES)
                {
                    // if the platform documentation token is being used, it's likely that it has expired and
                    // we need to get a new one from the user
                    SavedAccessToken.Default.DocsToken = null;
                    SavedAccessToken.Default.Save();

                    // try again... possibility for an infinite loop until stack overflow here, but since
                    // there's only one person running this that's fine for now.
                    this.Queue.Enqueue(item);
                    return;
                }

                if (ex.Response != null)
                {
                    try
                    {
                        using (var reader = new StreamReader(ex.Response.GetResponseStream()))
                        {
                            item.ErrorContent = reader.ReadToEnd();
                        }
                    }
                    catch { /* ignored */ }
                }

                throw;
            }

            if (!String.IsNullOrEmpty(content) && content != "{\"data\":[]}")
            {
                using (var file = new StreamWriter(item.CachedFile.Open(FileMode.Create, FileAccess.ReadWrite, FileShare.None)))
                {
                    using (var writer = new JsonTextWriter(file))
                    {
                        writer.WriteRaw(content);
                        writer.Flush();
                    }
                }

                JObject data;
                try { data = (JObject)JsonConvert.DeserializeObject(content); }
                catch (Exception ex) { item.ErrorContent = ex.Message; throw; }

                try { item.Callback.Invoke(data, item.Request.RequestUri); }
                catch (Exception ex) { item.ErrorContent = ex.Message; throw; }
            }
            else
            {
                var uri = item.Request.RequestUri;
            }
        }

        private MD5 MD5 = MD5.Create();
        private const Int32 MAX_RETRIES = 3;
        private void GetSampleData(String status, String url, Boolean useDocToken, Action<JObject, Uri> action) { GetSampleData(status, url, useDocToken, -1, action); }
        private void GetSampleData(String status, String url, Boolean useDocToken, Int32 retries, Action<JObject, Uri> action)
        {
            // use a hash to determine a unique name for the cached file
            var uri = new Uri(url);
            var urlID = MD5.ComputeHashString(url);
            var cachedFile = new FileInfo(String.Format("{0}{1}_{2}.txt", uri.Host, uri.AbsolutePath.Replace('/', '_'), urlID));

            // if there is no cached file, or the existing file is older than 1 hour, download new data from FBGraph
            if (!cachedFile.Exists || cachedFile.LastWriteTimeUtc < DateTime.UtcNow.AddHours(-24))
            {
                this.Queue.Enqueue(new ParserQueueItem(status, url, useDocToken, retries, cachedFile, action));                
            }
            else
            {
                var content = File.ReadAllText(cachedFile.FullName);

                JObject data;
                try { data = (JObject)JsonConvert.DeserializeObject(content); }
                catch (Exception ex) { throw; }

                try { action.Invoke(data, new Uri(url)); }
                catch (Exception ex) { throw; }
            }
        }
    }
}